{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model: K Nearest Neighbors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle as pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score, confusion_matrix, classification_report\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading in Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('../top10_features.xlsx')\n",
    "df = df.drop(df.columns[0], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaling the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "\n",
    "features_df = df.drop([\"Decision\"], 1)\n",
    "\n",
    "scaled_df = pd.DataFrame(scaler.fit_transform(features_df), \n",
    "                               index=features_df.index, \n",
    "                               columns=features_df.columns)\n",
    "\n",
    "df = scaled_df.join(df.Decision)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop([\"Decision\"], 1)\n",
    "y = df.Decision\n",
    "\n",
    "# Train, test, split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function for plotting confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_true, y_pred, labels=[\"Sell\", \"Buy\", \"Hold\"], \n",
    "                          normalize=False, title=None, cmap=plt.cm.coolwarm):\n",
    "\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    fig, ax = plt.subplots(figsize=(12,6))\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    # We want to show all ticks...\n",
    "    ax.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "           # ... and label them with the respective list entries\n",
    "           xticklabels=labels, yticklabels=labels,\n",
    "           title=title,\n",
    "           ylabel='ACTUAL',\n",
    "           xlabel='PREDICTED')\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 1.5\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, i, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"snow\" if cm[i, j] > thresh else \"orange\",\n",
    "                    size=26)\n",
    "    ax.grid(False)\n",
    "    fig.tight_layout()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling\n",
    "The preferred evaluation metric used will be __Precision__ for each class.  They will be optimized using the __F1 Score-Macro-Average__ to balance the Precision and Recall.  This is done because we want to not only be correct when predicting but also make a decent amount of predictions for each class.  Classes such as 'Buy' and 'Sell' are more important than 'Hold'."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fitting and Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing the model\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# Fitting and training\n",
    "clf = KNeighborsClassifier()\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Printing out Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        Sell       0.43      0.33      0.38         9\n",
      "         Buy       0.29      0.29      0.29         7\n",
      "        Hold       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.29        17\n",
      "   macro avg       0.24      0.21      0.22        17\n",
      "weighted avg       0.34      0.29      0.32        17\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Classifier predictions\n",
    "pred = clf.predict(X_test)\n",
    "\n",
    "#Printing out results\n",
    "report = classification_report(y_test, pred, target_names=['Sell', 'Buy', 'Hold'])\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAGoCAYAAABxHV2qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd7xcdZn48c9zW3KTkARCaAlFAVFAiiBKsRdQaT91BRUEu2tf3WXdXUXRtbvKqqy7WbtYWWUVxYKLdERBamgiEIEESQIJ6bc9vz9mgpfLvTe3zJ1zZubzfr3mxcyZ75x5yIE883zbicxEkiTVX1vRAUiS1KpMwpIkFcQkLElSQUzCkiQVxCQsSVJBTMKSJBXEJCyNICK6I+K8iFgdEedM4jyvjohf1TK2IkTEzyPilKLjkJqJSVgNLyJeFRFXR8TaiFhWTRZH1ODULwe2B+Zl5t9M9CSZ+e3MfGEN4nmUiHh2RGRE/GjI8f2rxy8a43k+FBFnb6ldZr4oM78xwXAlDcMkrIYWEe8BzgQ+RiVh7gL8B3BcDU6/K3B7ZvbV4FxTZTlwWETMG3TsFOD2Wn1BVPh3hTQF/B9LDSsi5gAfBt6WmT/KzHWZ2ZuZ52XmP1TbTIuIMyNiafVxZkRMq7737Ii4NyLeGxEPVKvo11bfOwM4HTihWmG/fmjFGBG7VSvOjurrUyPizohYExF3RcSrBx2/bNDnDouI31e7uX8fEYcNeu+iiPhIRFxePc+vImLbUf4YeoD/BU6sfr4deAXw7SF/Vv8eEfdExMMRcU1EPKN6/Cjgnwf9e14/KI6PRsTlwHrg8dVjb6i+/6WI+J9B5/9kRPxfRMSYL6Akk7Aa2qHAdODcUdr8C/B04ABgf+AQ4P2D3t8BmAMsAF4PnBURW2fmB6lU19/PzFmZ+ZXRAomImcDngRdl5lbAYcB1w7TbBvhZte084LPAz4ZUsq8CXgtsB3QBfz/adwPfBF5TfX4ksBhYOqTN76n8GWwDfAc4JyKmZ+Yvhvx77j/oMycDbwK2ApYMOd97gf2qPzCeQeXP7pR0H1xpXEzCamTzgBVb6C5+NfDhzHwgM5cDZ1BJLpv1Vt/vzczzgbXAXhOMZwDYNyK6M3NZZi4eps1LgD9m5rcysy8zvwvcChwzqM3XMvP2zNwA/IBK8hxRZl4BbBMRe1FJxt8cps3Zmbmy+p3/Bkxjy/+eX8/MxdXP9A4533rgJCo/Is4G3pGZ927hfJKGMAmrka0Ett3cHTyCnXh0FbekeuyRcwxJ4uuBWeMNJDPXAScAbwGWRcTPIuKJY4hnc0wLBr2+fwLxfAt4O/AchukZqHa531LtAl9FpfofrZsb4J7R3szM3wF3AkHlx4KkcTIJq5FdCWwEjh+lzVIqE6w224XHdtWO1TpgxqDXOwx+MzN/mZkvAHakUt3+9xji2RzTfROMabNvAW8Fzq9WqY+odhf/I5Wx4q0zcy6wmkryBBipC3nUruWIeBuVinopcNrEQ5dal0lYDSszV1OZPHVWRBwfETMiojMiXhQRn6o2+y7w/oiYX53gdDqV7tOJuA54ZkTsUp0U9k+b34iI7SPi2OrY8CYq3dr9w5zjfOAJ1WVVHRFxArA38NMJxgRAZt4FPIvKGPhQWwF9VGZSd0TE6cDsQe//BdhtPDOgI+IJwL9S6ZI+GTgtIkbtNpf0WCZhNbTM/CzwHiqTrZZT6UJ9O5UZw1BJFFcDNwA3An+oHpvId10AfL96rmt4dOJsozJZaSnwIJWE+NZhzrESOLradiWVCvLozFwxkZiGnPuyzByuyv8l8HMqy5aWUOk9GNzVvHkjkpUR8YctfU+1+/9s4JOZeX1m/pHKDOtvbZ55LmlswsmMkiQVw0pYkqSCjDarVJIkjVNE3A2soTIvpC8zDx6prUlYkqTae85Y5nrYHS1JUkGaamLWvLlzcpcdtys6DNXQ2ltH3S9CDWjtrvsUHYJqaPXKJaxfs6L0e4Yf1DYzH87hVg2Ozx1sWkxlhcFmizJz0eA2EXEX8BCVtfb/NfT9wZqqO3qXHbfj4m9+vugwVEOXHPruokNQjf32g1cUHYJq6CtnHLblRiXwcPZzZsfQfXLG7+i+2zeONsZbdXhmLo2I7YALIuLWzLxkuIZNlYQlSRpWQHTWoGAfw41NN6/Xz8wHIuJcKjeOGTYJOyYsSVKNRMTMiNhq83PghcBNI7W3EpYkNb2IoK2jLkPX2wPnVm+t3QF8p3rL0GGZhCVJzS8gOqe+8zcz76Ry7/IxMQlLkppfUK9KeFwcE5YkqSBWwpKk5ler2dE1ZhKWJDW9Ok7MGheTsCSp+ZW0EnZMWJKkglgJS5KaX0lnR5uEJUlNL4BoL18StjtakqSCWAlLkppfQFsJK2GTsCSpBQTRZhKWJKn+AqK9fCOw5YtIkqQWYSUsSWp6gWPCkiQVI3BMWJKkYkQpK2HHhCVJKoiVsCSp6UWUc8csk7AkqSVEW/k6f8sXkSRJLcJKWJLU/JwdLUlSUco5O9okLElqelHSStgxYUmSCmIlLElqCWWcHW0SliQ1v5J2R5uEJUktoJwTs8pXm0uS1CKshCVJTa+ss6NNwpKkluDELEmSilDSSrh8PwskSWoRVsKSpBYQpayETcKSpJZQxiRsd7QkSQWxEpYkNb3KEqXy1Z0mYUlSSyjjjlkmYUlS84tyTswqX20uSVKLsBKWJLUEx4QlSSqAe0dr3KJ3OR3rfkv7xttp77mb6FtFDKwGOhno3I6+7v3pnfsSBrp2LjpUjVH3LjvxvNt+tcV2fWvX84v5h9QhIk1eMj9uY2H8ngVtV7MwrmaHuJGO6KE3p3FGz+qiA1SVSVjj0rHut3Qv/9Iw7/TR3rOE9p4ldK0+n43z30jv3GPqHp8kmMsS3tV1QNFhqEGZhMssuuid8VT6Z+xP/7TdyY55ZPtsom8V7Rtvoeuhc2jvXUr38i+RndvTN9PKqZFcddxbePDya4Z/M+sbi2pjdS7g3oGDmRkr2K3t8qLD0aOEY8Ian945R9I758jHHM/22QxM24XeWYcza8nf0ta/kq6HfmgSbjADGzbRv25D0WFoktYzj7N7z+Hegaeylh0AeG77R0zCZVPSMeHy/SzQ2LXPonfW4ZWnG+8oOBipNfWwFbcOHPNIApbGw0q40UU7ANnWWXAgklRmdker1gZ66Fh3VeXptD0LDkYTFZ0dZG9f0WFIzS/K1x1tEm40OUD0r6J94+1Me/B7tPcuJaODjfNOKjoyjdM+n/tnZuy6gI5ZM+jfuIm1t/yJv/ziEu7+0nfoWf5g0eFJTcV1wpqUGfd9gI71j51J29+5kI3bvYOB6XsVEJUmY/Y+f+29aJ8+jTkH7s2cA/dmtze/kmtP+QeW//qKAqOTVA917SCPiH+JiMURcUNEXBcRTxul7dcj4uXV5xdFxMH1i7QxDLTPoWfucfRP36PoUDRGmQM88KvLuPb1/8zFBx3PL7Z7GudvczCXPP3l3HXW2Qz09dG1zRwO/v6/M/uAJxUdrtRUoq1t0o9aq1slHBGHAkcDT8nMTRGxLdBVr+9vdOt3/ADQD5nEwMN0bFhM10M/oHv5WXStPo/1O32I7HR2ZtltvOd+fnfcWx5z/OHrb2Xx9Z9gxcW/4+DvnUn7jG72+eRpXHnkawuIUmpC3kWJHYEVmbkJIDNXZObSiDgoIi6OiGsi4pcRsWMdY2ocbV3Q1g3tM8jOHeid/TzW7fx5+qbvRXvPn5mx9AzIgaKj1CT95bwLWfrDXwIw75lPZdqO8wuOSGoeZayE65mEfwXsHBG3R8R/RMSzIqIT+ALw8sw8CPgq8NHxnDQi3hQRV0fE1StXPTwFYZdY2zQ2zatUSu09S2jfcH3BAakWHjj/okeez9nvicUFImnK1a07OjPXRsRBwDOA5wDfB/4V2Be4ICpTx9uBZeM87yJgEcCBT9qz5Tb76x80Iat905/on3FggdGoFjY98NeZ0R1ztyowEqm5lLE7uq6zozOzH7gIuCgibgTeBizOzEPrGUdz6R/0vHz/gWn8pm2/7SPP+1atKTASqXmUdYlS3bqjI2KviBi8o8QBwC3A/OqkLSKiMyL2qVdMzaBjw02PPB9wYlZT2OGY5zzyfPUNtxYYidRMAtraJv+osXqOCc8CvhERN0fEDcDewOnAy4FPRsT1wHXAYXWMqdTaeu4ZvUH/Gqat+BoA2dZNX7dd0WU3faftRn1/x5cdyQ7HPR+AFRddxaZly+sRlqSC1HNM+BqGT7ArgGcO0/7UQc+fPWWBldjMJX9L38yn0TfrMPqn7UF2zCVpo61vJR0brqfroR/R1lf5S3rjvFOhfUaxAWuLnvHb/2Hlpb/nL+ddyOrrbmHTAyuJtjZmPmE3Fr7qWHZ57cuItjb61q5n8WmfKjpcjdH8uIVp/HVi6Oy4D4AgWRhXPartsjyAfqbVNT5VhNtWajyCATrXXUnnuitHbJPRxaZ5p9A795g6RqaJauvsYKeXHslOL33sLSo327j0L/zhlH9kzY231TEyTcaxHe/kcW2XPuZ4R/Twlq5nPerYZzbdyip2q1NkekTgDRw0PusWfor29dfTseEm2voeIPpWAb1k20wGunamv3s/ema/kOwcvYtT5XH9W05nm8Ofwtyn7sf0nbaja95coqOD3gdX8/CNt/HAzy/mnrN/TP/a9UWHKqkOTMIl1t+9L/3d+9JTdCCqmft//Gvu//Gviw5DNfaV3guKDkFbVL8dsyKiHbgauC8zjx6trUlYktT8gimZ3TyCd1FZ/TN7Sw3L10EuSdIUiLaY9GOL3xGxEHgJ8OWxxGQSliRp7LbdvFVy9fGmIe+fCZwGjGkzf7ujJUlNLwgialJ3rsjMYW+tGxFHAw9k5jUR8eyxnMwkLElqfgFM/cSsw4FjI+LFwHRgdkScnZknjfQBu6MlSS1hqm9lmJn/lJkLM3M34ETgwtESMJiEJUkqjN3RkqSWUM+7KGXmRVTuGjgqk7AkqflV7mVYdBSPUb6IJElqEVbCkqSWUM/u6LEyCUuSWoN3UZIkqf4iopT3Ey7fzwJJklqElbAkqTXYHS1JUjGcmCVJUhFcJyxJkgazEpYktQa7oyVJKkaN7idcUyZhSVLzq8/9hMetfD8LJElqEVbCkqQWEITrhCVJKojbVkqSpM2shCVJzS9w20pJkooRpeyONglLklpCGSdmlS8iSZJahJWwJKn5BaW8gYNJWJLUAqKUO2aZhCVJTS8o597R5YtIkqQWYSUsSWp+Jb2Bg0lYktQCopQTs8oXkSRJLcJKWJLUGtwxS5KkgpRwxyyTsCSp+YVjwpIkaRArYUlSa3CJkiRJBSlhd7RJWJLUGko4O7p8PwskSWoRVsKSpOYX4RIlSZIKU8LuaJOwJKk1lHBiVvkikiSpRVgJS5Kan2PC0vhtdd21RYegGnsB/UWHoBo6Z2YWHcLYlXBMuHw/CyRJahFWwpKk1lDCiVkmYUlSC4hSdkebhCVJzS8o5cSs8kUkSVKLsBKWJDW9BNLuaEmSihBOzJIkqTAlTMLli0iSpBZhJSxJagmOCUuSVIQo55hw+SKSJKlFWAlLklqD3dGSJBWkhDtmmYQlSS0gSjkxq3w/CyRJahFWwpKk5heUcna0SViS1BLSJCxJUhHKeT/h8v0skCSpQUXE9Ij4XURcHxGLI+KM0dpbCUuSWkKduqM3Ac/NzLUR0QlcFhE/z8zfDtfYJCxJag116I7OzATWVl92Vh85UnuTsCSp+dVu7+htI+LqQa8XZeaiR39VtAPXAHsAZ2XmVSOdzCQsSdLYrcjMg0drkJn9wAERMRc4NyL2zcybhmtrEpYkNb2k/rcyzMxVEXERcBQwbBJ2drQkqTVE2+QfW/qKiPnVCpiI6AaeD9w6UnsrYUmSamdH4BvVceE24AeZ+dORGpuEJUktIanL7OgbgAPH2t4kLElqAeG2lZIkFaaESbh8EUmS1CKshCVJzS/qv0RpLEzCkqSml44JS5JUoBJWwuX7WSBJUouwEpYktQS7oyVJKkTUZbOO8TIJS5JaQhkr4fJFJElSi7ASliQ1v6B5ZkdHxLtrHYgkSVMnSNom/ai1iZ7xPTWNQpKkFjTR7ujy1fSSJI0gaa5tK7OmUUiSNMXKODt6xCQcEWsYPtkGMGPKIpIkaQo01DrhzNyqnoHosaJ3OR3rfkv7xttp77mb6FtFDKwGOhno3I6+7v3pnfsSBrp2LjpUjUPbwAa2XnMB26z5NVttuIbuTXfRNrCOvva5rOvehxWzj+f+bU5moK276FA1Bl5PTca4uqMjYiZwPPCqzHzJ1ISkzTrW/Zbu5V8a5p0+2nuW0N6zhK7V57Nx/hvpnXtM3ePTxBx68+PoGFjzmONd/SvoWnsxW6+9mJ1W/ieLd/sBG6btUUCEGg+vZ6No0LsoRUQX8GLgVcBRwA+B/5ziuAQQXfTOeCr9M/anf9ruZMc8sn020beK9o230PXQObT3LqV7+ZfIzu3pm3lI0RFrDDoG1jAQ01g+51hWzj6aNd0H0dc+l2m997Djg19lp5VfZuam23jyncdy9RN+x0D7rKJD1ii8no2joSZmRcQLgFcCRwK/Ab4FHJKZr61TbC2vd86R9M458jHHs302A9N2oXfW4cxa8re09a+k66EfmoQbxH3z3syS7f6R3s7tH3W8r2Nr7lhwJps6d+bx959Od+8SFqxcxD3buSKwzLyejSEp55jwaLX5L4HdgSMy86TMPA8YqE9YGpP2WfTOOrzydOMdBQejsbpjwWcf8xf2YPfMfxe97fMA2GbNBfUKSxPk9dRkjNYdfRBwIvDriLgT+B7QXpeoNHZRuSTZ1llwIKqZ6GDDtN3pXL+Srt5lRUejyfJ6lkOUc0x4xIgy89rM/MfM3B34EHAg0BURP4+IN9UrQI1ioIeOdVdVnk7bs+BgVEudfQ8A0NfuIoVm4PUsh6zeznAyj1ob08+CzLw8M98OLADOBA6teSQamxwg+h6kY+1vmXnvabT3LiWjg43zTio6MtXIrA3X0d1zNwBrZjjO3+i8nhrNaBOznjLkUAIrMvOXVMaLVUcz7vsAHeuveczx/s6FbNzuHQxM36uAqDQVHr/sX4DKr/Zl2zgPstF5PcujjN3Ro40J/9swx7apLlk6MTOvH88XRUQ/cCOVHbf6gbdn5hXjOYcebaB9Dj1zj6N/umsPm8XCBz7H1msvAmDpvDeyrnvfYgPSpHg9y6WMs6NH2zHrOcMdj4iDgS8Azxznd23IzAOq5zgS+DjwrHGeo2Wt3/EDQD9kEgMP07FhMV0P/YDu5WfRtfo81u/0IbJzh6LD1CRsveYCHn//BwFYO30f7tzxYwVHpMnwepZLlnSzjnFHlJlXA5NdbT4beAggIp4dET/d/EZEfDEiTo2I50XEuYOOvyAifjTJ721cbV3Q1g3tM8jOHeid/TzW7fx5+qbvRXvPn5mx9AxIV5A1qlnrr2XvJScT9LOxcyE37fYjtzlsYF5PjdW4k3BEbM/E7qLUHRHXRcStwJeBj2yh/YXAkyJifvX1a4GvTeB7m1fbNDbNq4wxtfcsoX3DuEYIVBLdm/7Ik+86no6BNfS0b8sNj/sJm7oWFh2WJsjrWV5lnB092sSsL/DYZLsNcBjwrgl81+Du6EOBb0bEiAMkmZkR8S3gpIj4GpUZ2a8ZJs43AW8C2HmH7SYQVmPrHzQhq33Tn+ifcWCB0Wi8pvXcy353HkNX/wr62mZz4+P+lw1OsmtYXs9ya6htK4Grh7xOYCXwnsx8YDJfmplXRsS2wHygj0dX5NMHPf8acB6wETgnM/uGOdciYBHAgU/aswXvc9w/6Hn5/gPTyDr7lrPfXccwvfce+qObm3Y7h7X+iGpYXs/yyyzf35GjJeHnZOapU/GlEfFEKrtvrQSWAHtHxDQqCfh5wGUAmbk0IpYC7wdeMBWxNLqODTc98nzAiVkNo73/YZ581/HM2HQ7A9HJzbuezepZRxQdlibI66mJGi0J71fj7+qOiOuqzwM4JTP7gXsi4gfADcAfgWuHfO7bwPzMvLnG8ZReW889o98ruH8N01ZUhsmzrZu+bn91N4IY2Mi+d/8NW224jqSNW3f+Mg/OPqrosDRBXs9GEeT4p0FNudGS8IyIOJAR+jgz8w/j+aLMHHHf6cw8DThthLePAP57PN/VLGYu+Vv6Zj6NvlmH0T9tD7JjLkkbbX0r6dhwPV0P/Yi2vuUAbJx3KrTPKDZgbVn2s/efX8PcdZcB8KcdP8bKrY6irX/t8O2jjYE2r2tpeT0bRlnvojRaEl5AZcOO4aJO4LlTEtEgEXENsA5471R/VxkFA3Suu5LOdVeO2Caji03zTqF37jF1jEwTNa33XrZ9+GePvN5j2fvYY9n7Rmy/sXMXrnrSLfUITRPg9WwsjZaE78jMKU+0o8nMg4r8/qKtW/gp2tdfT8eGm2jre4DoWwX0km0zGejamf7u/eiZ/UKys/VmhUtSMxgtCatg/d370t+9Lz1FB6Ka2dS1Kxfvt67oMFQjXs/GUsZKeLRR6o9HxN5DD0bEPoM20JAkqQFMfqOOet/K8KVU1vEOtRD495pHIklSixktCT85My8eerB6K8NaL1+SJGlKZcakH7U22phw1yjvddY6EEmSpkpZlyiNVgnfHhEvHnowIl4E3Dl1IUmSVHtlHBMerRL+O+CnEfEK4JrqsYOp3Ejh6JpHIklSixmxEs7M24EnAxcDuwG7AhcBr2Nid1GSJKkwjVYJk5mbgK9Vt698JfBB4C7ghzWPRJKkKTM1E6sma7T7CT8BOJFK8l0JfB+IzHxOnWKTJKkmEhgo4cSs0SrhW4FLgWMy8w6AiPi7ukQlSVILGG129MuA+4HfRMR/R8Tz8K7xkqQGVcYx4dEmZp2bmScAT6QyIevvgO0j4ksR8cKaRyJJ0lTJcm7WscU7HGfmusz8dmYeTWXLyuuAke/VJUmSxmRcd1HKzAeB/6o+JElqGGXcMctbGUqSWkCDLVGSJKlZNOLe0ZIkaQpZCUuSWoLd0ZIkFWSg6ACGYRKWJLWEMlbCjglLklQQK2FJUtObqm0nJ8skLElqCXZHS5JUkHrcwCEido6I30TELRGxOCLeNVp7K2FJkmqnD3hvZv4hIrYCromICzLz5uEam4QlSc0vYSDr8DWZy4Bl1edrIuIWYAFgEpYktaYitq2MiN2AA4GrRmpjEpYkaey2jYirB71elJmLhjaKiFnAD4F3Z+bDI53MJCxJagk1mh29IjMPHq1BRHRSScDfzswfjdbWJCxJaglZhzHhiAjgK8AtmfnZLbV3iZIkqQUEAzV4jMHhwMnAcyPiuurjxSM1thKWJKlGMvMyGPsMMJOwJKnpJeXcMcskLElqCfUYEx4vk7AkqSWU8QYOTsySJKkgVsKSpOZXp20rx8skLElqemWdmGV3tCRJBbESliS1BGdHS5JUkDHueFVXJmFJUksoYyXsmLAkSQWxEpYkNb0kSjk72iQsSWp+rhOWJKk4jglLkqRHWAlLklpCGW/gYBKWJDW9xDFhSZIKU8Yx4aZKwvdvmM2nbnpu0WGohn7zo98WHYKkUdx337qiQ2hoTZWEJUkaiZWwJEkFyISBEm7W4RIlSZIKYiUsSWoJdkdLklQQk7AkSQUp4zphx4QlSSqIlbAkqekleCtDSZIKkY4JS5JUGMeEJUnSI6yEJUlNrzImXHQUj2USliS1hDImYbujJUkqiJWwJKkllHFilklYktT8XKIkSVIxEhgYKDqKx3JMWJKkglgJS5Jagt3RkiQVxCQsSVIBMss5O9oxYUmSCmIlLElqCVnC/miTsCSpJZQwB5uEJUmtwXXCkiTpEVbCkqSml25bKUlScVyiJEmSHmElLElqCXZHS5JUkCxhf7RJWJLU9Ny2UpIkPYqVsCSpJTgmLElSQQZK2B9tEpYkNb2knJWwY8KSJBXESliS1PzctlKSpKIkAyXMwnZHS5JUECthSVJLyBLeT9gkLElqepXZ0XZHS5JUfwkDA5N/bElEfDUiHoiIm8YSlklYkqTa+Tpw1Fgb2x0tSWoJ9eiOzsxLImK3sbY3CZdeMj9uY2H8ngVtV7MwrmaHuJGO6KE3p3FGz+qiA9QkPGPBpRy/5094wtZ3MLNzLSs2bMtVyw7he7e+gvvWLig6PI2T17O8knLeRckkXHJzWcK7ug4oOgzVXPK+Qz7NMbuf/6ijC2Yt46V7/pijdvsVp1/+Qa5c9vSC4tP4eD1LL2t2P+FtI+LqQa8XZeaiiZ7MJNxAVucC7h04mJmxgt3aLi86HE3Ca/b+9iN/Yf/fkufwjZtPYuWGeey77WLe9ZQvsNOs+/nw4Wfwul8u4p41OxccrbbE69lSVmTmwbU6mROzSm498zi79xw+seluPt3zJ77b933uHHh20WFpErae/iAn7302AJff93ROv+J0/rRqd1Ztmstl9x3OOy/8HOt7pzOjcwNv2u/LBUerLfF6No7MyT9qzSRccj1sxa0Dx7CWHYoORTXyot1+xYzOjQAsuuENQDzq/WXrduS8Px0NwLMWXsrW0x+sd4gaB69n4xgYyEk/tiQivgtcCewVEfdGxOtHa28Slurs8AVXAPDnhxdyx6o9hm3zm3ueBUB72wCH7nhV3WLT+Hk9G0Nm1uQxhu95ZWbumJmdmbkwM78yWnuTsFRnT9j6jwDcvPJJI7a59cG96Buo/O+51za31yUuTYzXU5NhEpbqaNvu5czo3ADA0rU7jdiud6CLlRvmAbDr7D/XJTaNn9ezseTA5B+1ZhKW6mjutL+u6161ac6obR/atDUAs7sentKYNHFez8YykDnpR61NWRKOiLVDXp8aEV/cwmc+FBF/P8zx3ca6D6dUZtM7Nj7yvKe/a9S2m6rvd3dsmNKYNHFeT02W64SlOgr++ks6h8yifWzbxz5TuXg9G0sZ76JUSBKOiF2BrwLzgeXAazPzz0PaHFRtsx64rO5BSlNgQ1/3I8+ntW8atW1Xe0/1M9OnNCZNnNezcWQypiVG9TaVY8LdEXHd5gfw4UHvfRH4ZmbuB3wb+Pwwn/8a8M7MPHS0L4mIN0XE1QxvwvIAAA0JSURBVBFx9fq1y2sWvDQVBo8bDh5PHM7caasAeLhn9pTGpInzejaWVtusY0NmHrD5AZw+6L1Dge9Un38LOGLwByNiDjA3My8e1GZYmbkoMw/OzINnzJpfw/Cl2luxYT7reyvV046zlo3YrrOth227VwCw5OFd6hKbxs/rqckqy+zoob8vYphjUlO4/aE9Adhn3i0jttlrm9vpaKush7jtwSfUJS5NjNezceRATvpRa0Ul4SuAE6vPX82QMd/MXAWsjogjBrWRmsLl9x0GwC6z7+Hxc+4cts1zdq50AvUPtHHlsqfVLTaNn9ezMWQNlic11BKlLXgn8NqIuAE4GXjXMG1eC5wVEVcCzulX0/j53S98ZHLOm4fZ0H+Hmcs4dvfzALj43mfw0MZt6hqfxsfr2TjKWAlP2ezozJw15PXXga9Xn98NPHeYz3xo0PNrgP0Hvf2hoe1bxfy4hWn8dYH/7LgPqCyPWBiP3od2WR5AP9PqGp/G56GN2/DNxSfx5v2/zBELr+CMw87gG4tP5sGN27D3vJt590FfYEbnRtb3dldvCKAy83pqMlwn3ACO7Xgnj2u79DHHO6KHt3Q961HHPrPpVlaxW50i00R98+ZXs9OspRyz+/k8f9ff8Pxdf/Oo99f3dnP65R/03rMNwuvZGKaikp0sk7BUiOATvzuNK5YeynF7/IS9tv4jMzrXsWLDtvxu2VP57q0ncN/aBUUHqTHzepZeQglzsEm4EXyl94KiQ9AUueTeZ3DJvc8oOgzViNdT42USliQ1vcTuaEmSCpLuHS1JUiFacO9oSZI0CithSVJLsDtakqQCODFLkqSiZDmTsGPCkiQVxEpYktQCpuYuSJNlEpYktYQydkebhCVJTS8p5+xox4QlSSqIlbAkqfmVdMcsk7AkqSWUcUzY7mhJkgpiJSxJagHeRUmSpEJkQg4MFB3GY5iEJUktoYwTsxwTliSpIFbCkqSW4JiwJElFyCzlEiWTsCSp6ZX1fsKOCUuSVBArYUlSSxhIlyhJklR/aXe0JEkaxEpYktT0EmdHS5JUGNcJS5JUhISBEu4d7ZiwJEkFsRKWJLUEx4QlSSpAkqTrhCVJKoDrhCVJ0mBWwpKkllDGStgkLElqAene0ZIkFSEdE5YkSYNZCUuSWkKWcMcsk7AkqfnZHS1JkgazEpYktQB3zJIkqRAJDJSwO9okLElqflnOiVmOCUuSVBArYUlSC0hnR0uSVJTMgUk/tiQijoqI2yLijoh435baWwlLkppfHdYJR0Q7cBbwAuBe4PcR8ZPMvHmkz1gJS5JUG4cAd2TmnZnZA3wPOG60D1gJS5KaXpL1mB29ALhn0Ot7gaeN9oHILN9A9URFxHJgSdFx1MG2wIqig1BNeU2bT6tc010zc37RQWxJRPyCyjWZrOnAxkGvF2Xmoup3/A1wZGa+ofr6ZOCQzHzHSCdrqkq4Ef5DqIWIuDozDy46DtWO17T5eE3LJTOPqsPX3AvsPOj1QmDpaB9wTFiSpNr4PbBnRDwuIrqAE4GfjPaBpqqEJUkqSmb2RcTbgV8C7cBXM3PxaJ8xCTemRUUHoJrzmjYfr2kLyszzgfPH2r6pJmZJktRIHBOWJKkgJmFJkgpiEm4wERFFxyBpZBExt+gY1DhMwg0kIp4KvCYiuouORbUXEU6UbHARsQC4PCKeW3Qsagwm4cYyE3g78LKImF50MKqdiHgC8KWImFZ0LJqYiIjMvA/4NPDpiHh60TGp/EzCDSAinhwRJ2fmRcB7gTcArzARN75BwwtdwACVtYVqMNUEvHmpyd1Uts9dFBGHFReVGoFJuDE8Gfh/EfGqzLwE+BDwOkzEzWB29Z+3AdsDHywwFk3Q5gRc3ajho1TunnMh8J8R8cwiY1O5mYRLbHOVlJnfAc4BXhIRJ1Ur4g9RScQvd4y4MUXEQuCbEfH6zOylMtQwMyJ2LTg0jVFE7BURLxp0aE/gA5n5A+A9wH8Cn4uIIwoJUKXnRJCSGtK9RWZ+NyIeBk6OCDLz7Ig4Hfg80At8v6hYNX4RsQuV2559Fvj7iNiPSiXcDTwRWDL0vwGVS0R0Ai8DFlQv1S+oXL+TgAszcyAiLgReCXwiIl6QmRsKDFkl5I5ZJRcRbwR2oXLrrLOAI6hsCn5+Zn4nIg4H7s3MVriFY8OLiDZgDvAJKvcd/TcggLlUxvufD6wBXp6Z9xcVp8YmInYAXgPsSOWH8C1Utiy8KjPfExEnAPsCX8zMvxQXqcrKJFwyETEjM9dXn78TOBb4MHAm8MPM/Gj1npWvAb5V7fZSyQ2taqtLWP4GuBM4NzPvqB7fB3gz8JXMvL6QYDWqYa7lfCpDQztTGQv+I/BDKrewezKVH1SjbuKv1uWYcIlExIuBj0XEzhHRTuV/6iOBg4H7qSx76MrMc4D/Ai4vLlqNR2ZmROwfEV+ovr4Q+C6VMcQTIuLx1eOLqdyD9IWFBasRDU7AEXFMRBwF7JWZn6QyI/oEYJfMPAI4FTjCBKzRmIRLIiKOBj4OXJSZ91BZrrIQuIhKF/RxmdkDvC4ijs3Mn1bXJKqkImL3iHhpRBxfPdQLbBMRn6v+ZX4J8DPgLcBLI2JuRMyk0jU95ruwqP4i4q1UeqiOAP47Iv4lMz9NZXnSWyLieZm5PjNXFhmnys8kXALVcaX3Am/IzP+NiOnVX9tfpzLWdHZm9kbEqcC7AH9Zl1x1840fA4cDp0XE6zLzZirLV+ZQGV4AuB64FvhFZq7KzHXAi6yeyiUi9oiIOdUeje2oDCW8KjPfDxxG5cfxqcCXgZuAG4uLVo3E2dHlsIlKlbSxuu73fRHxLCoTdB6ksuj/RcABwMsy80/FhaotiYi9gW8D/5SZ50XEScDsiNgnMxdHxKeAj0bElVSq3ndn5k2Dujp7CgxfQ0TE1sDbgJ6I+HhmPhARK6lep8x8KCL+Djg8M78eEZ/PzP4iY1bjcGJWCVTXA7+HyjjgPsCvgcuAm4HjgduBc4G2zFxeVJwam+qa0Esys636+gbgPmAn4NrMPLV6/CXAfZl5XVGxamSbfxRV//88ikrF2w+cAXwMeAHw9Mzsi4h3AE+nMmFywKVlGisr4RKo/o/+X8AVVCZj/TgzNwFExJuAGxxbahyZeVlEvCQi7qQy+/l/MvPDEdEF3BgR78/Mf83MnxUcqkbXDvRRKVZ+HhGzgdOAdZn5TxGxFXBJ9UfW04BXWwFrvKyES6y6FOl9wCvsgm48EfE84JdAV2YOVI+9Hpibmf9WaHAaVURsC1wNHFLtft6Jyq511wNrgYcy8+MR8RQqY/x3Z+ZdxUWsRuXErBKKiB0j4t1UtqY8xQTcmDLz/6is874dKpN7gH/ASTull5krgHcAF0bEvsC3gO9k5lupzFzfLiI+CdyRmb8xAWui7I4up1VUFvwft3kTBzWmzDw/IgYiYj1wF5VJWL8qOi5tWXVSXS9wA/DPmXlW9a1LgWnAM6r/lCbM7mipDqpd07Mz89yiY9H4RMQLgC8AT8vM1YOOP7K7nTRRJmGpjrwpQ2OqLhE8Ezg0Mx8sOh41D7ujpToyATem6uzoLuDXEXFw5ZDXUpNnJSxJYxQRszJzbdFxqHmYhCVJKohLlCRJKohJWJKkgpiEJUkqiElYkqSCuERJGkFE9FPZYrIDuIXKFqLrhxy/Czg5M1dFxG7VdrcNOs1nM/ObEXE3lVtTQuXGAD8CPpKZm6qf+2lm7lv93kOAzwDbA0nljlrXAm+sfn7v6nf0A78AbgU+TeVOTZu9ClhfjedWYHr1+8/KzG9M8o9GUo04O1oaQUSszcxZ1effBq7JzM8OOf4N4PbM/OjQZDrkXHcDB2fmioiYBSwCejPzlMGfi4jtgd8BJ2bmldXb6L0MuDQz/zL0XNXXp1Zfv33Idz4qnoh4PJXk/++Z+bUa/TFJmgS7o6WxuRTYY5jjVwILxnOi6jrTtwDHR8Q2Q95+G/CNzLyy2jYz8382J+DJyMw7qdy3+p2TPZek2jAJS1sQER3Aixhy96OIaAeeB/xk0OHdI+K6QY9nDHfOzHyYSlf2nkPe2he4ZgJhnjDke7tHaPcH4IkTOL+kKeCYsDSy7oi4rvr8UuArQ47vRiVhXjDoM3/KzAPGeP6oSZQV3x+mO3qqv1PSJFkJSyPbkJkHVB/vyMyewceBXYEuKl3I4xIRW1FJ4rcPeWsxcNAkYt6SA6lM1pJUAiZhaYKqt7V7J/D3EdE51s9VJ2b9B/C/mfnQkLe/CJwSEU8b1P6kiNhhsvFWJ2p9hspt+SSVgN3R0iRk5rURcT1wIpUu690HdWEDfDUzP199/pvqbOc24FzgI8Oc7y8RcSLwmYjYDhgALqEyq3k0J0TEEYNevxVYWo3nWv66ROkLzoyWysMlSpIkFcTuaEmSCmISliSpICZhSZIKYhKWJKkgJmFJkgpiEpYkqSAmYUmSCvL/ATXwg4HYmMsgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(y_test, pred, title=\"Confusion Matrix\")\n",
    "np.set_printoptions(precision=1)\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tuning Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters to Tune\n",
    "params = {\"n_neighbors\": [10, 15],\n",
    "          \"weights\": ['uniform', 'distance'],\n",
    "          \"p\": [1,2]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "[CV] n_neighbors=10, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=1, weights=uniform, score=(train=0.354, test=0.372), total=   0.0s\n",
      "[CV] n_neighbors=10, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=1, weights=uniform, score=(train=0.581, test=0.360), total=   0.0s\n",
      "[CV] n_neighbors=10, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=1, weights=uniform, score=(train=0.610, test=0.403), total=   0.0s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=10, p=1, weights=distance, score=(train=1.000, test=0.327), total=   0.0s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=10, p=1, weights=distance, score=(train=1.000, test=0.430), total=   0.0s\n",
      "[CV] n_neighbors=10, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=10, p=1, weights=distance, score=(train=1.000, test=0.403), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=2, weights=uniform, score=(train=0.347, test=0.317), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=2, weights=uniform, score=(train=0.312, test=0.228), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=10, p=2, weights=uniform, score=(train=0.470, test=0.519), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n",
      "[CV]  n_neighbors=10, p=2, weights=distance, score=(train=1.000, test=0.316), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_neighbors=10, p=2, weights=distance, score=(train=1.000, test=0.190), total=   0.0s\n",
      "[CV] n_neighbors=10, p=2, weights=distance ...........................\n",
      "[CV]  n_neighbors=10, p=2, weights=distance, score=(train=1.000, test=0.486), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=1, weights=uniform, score=(train=0.355, test=0.099), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=1, weights=uniform, score=(train=0.362, test=0.292), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=1, weights=uniform, score=(train=0.385, test=0.240), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=15, p=1, weights=distance, score=(train=1.000, test=0.138), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=15, p=1, weights=distance, score=(train=1.000, test=0.348), total=   0.0s\n",
      "[CV] n_neighbors=15, p=1, weights=distance ...........................\n",
      "[CV]  n_neighbors=15, p=1, weights=distance, score=(train=1.000, test=0.263), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=2, weights=uniform, score=(train=0.339, test=0.262), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=2, weights=uniform, score=(train=0.395, test=0.198), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=uniform ............................\n",
      "[CV]  n_neighbors=15, p=2, weights=uniform, score=(train=0.480, test=0.328), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=distance ...........................\n",
      "[CV]  n_neighbors=15, p=2, weights=distance, score=(train=1.000, test=0.344), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=distance ...........................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_neighbors=15, p=2, weights=distance, score=(train=1.000, test=0.184), total=   0.0s\n",
      "[CV] n_neighbors=15, p=2, weights=distance ...........................\n",
      "[CV]  n_neighbors=15, p=2, weights=distance, score=(train=1.000, test=0.337), total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.3s finished\n",
      "C:\\Users\\72445\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=None,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'n_neighbors': [10, 15], 'p': [1, 2],\n",
       "                         'weights': ['uniform', 'distance']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='f1_macro', verbose=5)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search = GridSearchCV(clf, params, cv=3, return_train_score=True, verbose=5, scoring='f1_macro')\n",
    "\n",
    "search.fit(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuned Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Training Score: 0.7079172203975561\n",
      "Mean Testing Score: 1.0\n",
      "\n",
      "Best Parameter Found:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 10, 'p': 1, 'weights': 'distance'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Mean Training Score:\", np.mean(search.cv_results_['mean_train_score']))\n",
    "print(\"Mean Testing Score:\", search.score(X, y))\n",
    "print(\"\\nBest Parameter Found:\")\n",
    "search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model with the Best Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=10, p=1,\n",
       "                     weights='distance')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_clf = search.best_estimator_\n",
    "\n",
    "search_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Results from Optimum Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        Sell       0.50      0.22      0.31         9\n",
      "         Buy       0.17      0.14      0.15         7\n",
      "        Hold       0.14      1.00      0.25         1\n",
      "\n",
      "    accuracy                           0.24        17\n",
      "   macro avg       0.27      0.46      0.24        17\n",
      "weighted avg       0.34      0.24      0.24        17\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Classifier predictions\n",
    "s_pred = search_clf.predict(X_test)\n",
    "\n",
    "#Printing out results\n",
    "report = classification_report(y_test, s_pred, target_names=['Sell', 'Buy', 'Hold'])\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion Matrix for Optimum Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAGoCAYAAABxHV2qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZhcZZnw/+/dSXfSIQlJSICQAGERlE2UiLKpuAEKyIsLjIKCOsjPDUfnxfWHy4zb6CiKzDhxV9BxAxVFEQdZZdAga9hEFgMJQhIge9JJ3+8fVYmdprvTS3WdU1Xfz3XVRdWpp865k0PlrmePzESSJNVfW9EBSJLUqkzCkiQVxCQsSVJBTMKSJBXEJCxJUkFMwpIkFcQkLPUjIjoj4pKIeDIifjSC87w+In5Ty9iKEBG/iog3Fh2H1ExMwmp4EfG6iJgfESsjYnE1WRxeg1O/GtgB2C4zXzPck2TmhZn5shrEs4WIeGFEZERc1Ov4M6vHrxzkeT4aERdsrVxmHpOZ3x5muJL6YBJWQ4uI9wDnAp+kkjB3Af4DeGUNTr8rcE9mbqjBuUbLY8ChEbFdj2NvBO6p1QWiwn8rpFHgF0sNKyK2BT4OvD0zL8rMVZnZlZmXZOb/rZYZFxHnRsSi6uPciBhXfe+FEfFQRLw3Ih6t1qJPr773MeAc4KRqDfvNvWuMETGnWuMcW319WkTcFxErIuL+iHh9j+PX9vjcoRHxx2oz9x8j4tAe710ZEf8SEddVz/ObiJg+wF/DeuCnwMnVz48BXgtc2Ovv6osRsTAilkfEjRFxRPX40cAHe/w5b+kRxyci4jpgNbB79dhbqu//Z0T8uMf5PxMR/xMRMegbKMkkrIZ2CDAeuHiAMh8CngccCDwTOBj4cI/3dwS2BWYBbwbOj4ipmfkRKrXrH2TmxMz8+kCBRMQ2wJeAYzJzEnAocHMf5aYBv6yW3Q74PPDLXjXZ1wGnA9sDHcA/D3Rt4DvAG6rPjwIWAIt6lfkjlb+DacD3gB9FxPjM/HWvP+cze3zmVOAMYBLwYK/zvRc4oPoD4wgqf3dvTNfBlYbEJKxGth2wZCvNxa8HPp6Zj2bmY8DHqCSXTbqq73dl5qXASmDvYcbTDewXEZ2ZuTgzF/RR5hXAnzPzu5m5ITO/D9wFHNejzDcz857MXAP8kEry7Fdm/h6YFhF7U0nG3+mjzAWZubR6zX8HxrH1P+e3MnNB9TNdvc63GjiFyo+IC4B3ZuZDWzmfpF5MwmpkS4Hpm5qD+7ETW9biHqwe23yOXkl8NTBxqIFk5irgJOBMYHFE/DIinj6IeDbFNKvH60eGEc93gXcAR9JHy0C1yf3OahP4E1Rq/wM1cwMsHOjNzPwDcB8QVH4sSBoik7Aa2fXAWuCEAcosojLAapNdeGpT7WCtAib0eL1jzzcz87LMfCkwk0rt9quDiGdTTA8PM6ZNvgu8Dbi0WkvdrNpc/D4qfcVTM3MK8CSV5AnQXxPygE3LEfF2KjXqRcDZww9dal0mYTWszHySyuCp8yPihIiYEBHtEXFMRPxbtdj3gQ9HxIzqAKdzqDSfDsfNwPMjYpfqoLAPbHojInaIiOOrfcPrqDRrb+zjHJcCe1WnVY2NiJOAfYBfDDMmADLzfuAFVPrAe5sEbKAyknpsRJwDTO7x/t+AOUMZAR0RewH/SqVJ+lTg7IgYsNlc0lOZhNXQMvPzwHuoDLZ6jEoT6juojBiGSqKYD9wK3Ab8qXpsONe6HPhB9Vw3smXibKMyWGkRsIxKQnxbH+dYChxbLbuUSg3y2MxcMpyYep372szsq5Z/GfArKtOWHqTSetCzqXnTQiRLI+JPW7tOtfn/AuAzmXlLZv6Zygjr724aeS5pcMLBjJIkFcOasCRJBRloVKkkSRqiiHgAWEFlXMiGzJzbX1mTsCRJtXfkYMZ62BwtSVJBmmpg1rZTp+eOO+1SdBiqoe477ig6BNVY2z77FB2CauiRRX/lyceXlH7N8IPatsnl2deswaG5l3ULqMww2GReZs7rWSYi7gcepzLX/r96v99TUzVH77jTLsz7wbVbL6iGseLAZxUdgmpskt/RpnLGSbXYNXT0Lc+NnDu29zo5Q3fshnvWDtTHW3VYZi6KiO2ByyPirsy8uq+CTZWEJUnqU0C016DCPoiNTTfN18/MRyPiYiobx/SZhO0TliSpRiJim4iYtOk58DLg9v7KWxOWJDW9iKBtbF26rncALq5urT0W+F51y9A+mYQlSc0vINpHv/E3M++jsnf5oJiEJUnNL6hXTXhI7BOWJKkg1oQlSc2vVqOja8wkLElqenUcmDUkJmFJUvMraU3YPmFJkgpiTViS1PxKOjraJCxJanoBxJjyJWGboyVJKog1YUlS8wtoK2FN2CQsSWoBQbSZhCVJqr+AGFO+HtjyRSRJUouwJixJanqBfcKSJBUjsE9YkqRiRClrwvYJS5JUEGvCkqSmF1HOFbNMwpKklhBt5Wv8LV9EkiS1CGvCkqTm5+hoSZKKUs7R0SZhSVLTi5LWhO0TliSpINaEJUktoYyjo03CkqTmV9LmaJOwJKkFlHNgVvnq5pIktQhrwpKkplfW0dEmYUlSS3BgliRJRShpTbh8PwskSWoR1oQlSS0gSlkTNglLklpCGZOwzdGSJBXEmrAkqelVpiiVr95pEpYktYQyrphlEpYkNb8o58Cs8tXNJUlqEdaEJUktwT5hSZIK4NrRGpa27jVMXXE501b8lklrbqRz3f20da9iw5gprOrclyWTT+CRaafS3dZZdKgahM5dduLFd/9mq+U2rFzNr2ccXIeINFJ+RxuHSVhDdsgduzG2e8VTjndsXELHyquYuvIqdlr6FRbM+SFrxu1ZQIRSa/M7qpEwCZfc2O4VdMc4Htv2eJZOPpYVnQexYcwUxnUtZOayb7DT0q+xzbq72f++45m/1x/oHjOx6JA1SDe88kyWXXdj329mfWPR8PkdbRRhn7CG7uHt3sqD27+PrvYdtji+YexU7p11Luvad2b3R86hs+tBZi2dx8Lt31NQpBqq7jXr2LhqTdFhaIT8jjaIkvYJl+9ngbZw76zPP+XL3dPCGWfRNWY7AKatuLxeYUmq8juqkbAm3OhiLGvG7UH76qV0dC0uOhpJvfkdLQmbozVK2jc8CsCGMZMKjkTDEe1jya4NRYehUeR3tCSifM3RJuEGN3HNzXSufwCAFROc0tJI9v3CB5mw6yzGTpzAxrXrWHnnX/jbr6/mgf/8HusfW1Z0eKoRv6PlUNZ5wuWrm2tIdl/8IQCSYPG00wuORkMxed+nMXbiBADGjB/Hts/ah70+cCYvvPkSZrzk0IKjU634HdVA6pqEI+JDEbEgIm6NiJsj4rkDlP1WRLy6+vzKiJhbv0gbw+xHv8DUlVcCsGi7f2RV537FBqStyuzm0d9cy01v/iBXHXQCv97+uVw6bS5XP+/V3H/+BXRv2EDHtG2Z+4MvMvnAZxQdrkbI72i5RFvbiB+1Vrfm6Ig4BDgWeHZmrouI6UBHva7fbKauuJzdH/kIACvH78t9Mz9ZcEQajLULH+EPrzzzKceX33IXC275NEuu+gNz//tcxkzoZN/PnM31R1lzalR+R0vGXZSYCSzJzHUAmbkkMxdFxEERcVVE3BgRl0XEzDrG1JAmrr6JfR48lWAja9tnc/uci1wSr0n87ZIrWPSTywDY7vnPYdzMGQVHpOHwO1pOZawJ1zMJ/wbYOSLuiYj/iIgXREQ7cB7w6sw8CPgG8ImhnDQizoiI+REx/8nHl4xC2OXSue7P7H//CYztXsH6MdO5dbefs65jdtFhqYYevfTKzc+3PeDpxQWiYfE7qqGoW3N0Zq6MiIOAI4AjgR8A/wrsB1welaHjY4AhTaTLzHnAPIC99312Uy/2N279Qxxw33F0bFzChrbJ3LbbT1kzfu+iw1KNrXv07yOjx05xSksj8TtabmVsjq7rFKXM3AhcCVwZEbcBbwcWZOYh9YyjEbVveIwD7j+O8V0L2Rid3D7nR6yc8Kyiw9IoGLfD9M3PNzzx1I0BVE5+R8ut5acoRcTeEfG0HocOBO4EZlQHbRER7RGxb71iahRjNi5n//tPYMK6e+iOdu7Y9QKenHh40WFplOx43JGbnz95610FRqLB8jvaCALa2kb+qLF61oQnAudFxBRgA3AvcAaVpuQvRcS21XjOBRbUMa5Si+617PfAa5i05maSNu7a+Wssm3x00WFpmMbvtD1rFz3a7/szX3UUO77yJQAsufIG1i1+rF6haZj8jmok6tknfCPQ1woES4Dn91H+tB7PXzhqgZVZbmSfv76BKauuBeAvMz/J0klH07ZxZd/lo43utgl1DFBDdcT//pil1/yRv11yBU/efCfrHl1KtLWxzV5zmP2649nl9FcRbW1sWLmaBWf/W9Hhamv8jjaUcNlKDcW4roeYvvyXm1/vufj97Ln4/f2WX9u+Czc84856hKZhamsfy04nHsVOJx7Vb5m1i/7Gn974PlbcdncdI9Nw+B1tIIEbOEit7pYzz2HaYc9mynMOYPxO29Ox3RRi7Fi6lj3J8tvu5tFfXcXCC37GxpWriw5VUh2YhEtsXceuXHXAqqLDUA098rPf8sjPflt0GKoRv6ONpH4rZkXEGGA+8HBmHjtQWZOwJKn5BaMyurkfZ1GZ/TN5awXL10AuSdIoiLYY8WOr14iYDbwC+NpgYjIJS5I0eNM3LZVcfZzR6/1zgbOB7sGczOZoSVLTC4KImtQ7l2Rmn1vrRsSxwKOZeWNEvHAwJzMJS5KaXwCjPzDrMOD4iHg5MB6YHBEXZOYp/X3A5mhJUksY7a0MM/MDmTk7M+cAJwNXDJSAwSQsSVJhbI6WJLWEeu6ilJlXUtk1cEAmYUlS86vsZVh0FE9RvogkSWoR1oQlSS2hns3Rg2USliS1BndRkiSp/iKilPsJl+9ngSRJLcKasCSpNdgcLUlSMRyYJUlSEZwnLEmSerImLElqDTZHS5JUjBrtJ1xTJmFJUvOrz37CQ1a+nwWSJLUIa8KSpBYQhPOEJUkqiMtWSpKkTawJS5KaX+CylZIkFSNK2RxtEpYktYQyDswqX0SSJLUIa8KSpOYXlHIDB5OwJKkFRClXzDIJS5KaXlDOtaPLF5EkSS3CmrAkqfmVdAMHk7AkqQVEKQdmlS8iSZJahDVhSVJrcMUsSZIKUsIVs0zCkqTmF/YJS5KkHqwJS5Jag1OUJEkqSAmbo03CkqTWUMLR0eX7WSBJUouwJixJan4RTlGSJKkwJWyONglLklpDCQdmlS8iSZJahDVhSVLzs09YGrr//eqtRYegGnvPz84qOgTVUOcTC4sOYfBK2Cdcvp8FkiS1CGvCkqTWUMKBWSZhSVILiFI2R5uEJUnNLyjlwKzyRSRJUouwJixJanoJpM3RkiQVIRyYJUlSYUqYhMsXkSRJLcKasCSpJdgnLElSEaKcfcLli0iSpBZhTViS1BpsjpYkqSAlXDHLJCxJagFRyoFZ5ftZIElSi7AmLElqfkEpR0ebhCVJLSFNwpIkFaGc+wmX72eBJEkNKiLGR8QfIuKWiFgQER8bqLw1YUlSS6hTc/Q64EWZuTIi2oFrI+JXmfm/fRU2CUuSWkMdmqMzM4GV1Zft1Uf2V94kLElqfrVbO3p6RMzv8XpeZs7b8lIxBrgR2BM4PzNv6O9kJmFJkgZvSWbOHahAZm4EDoyIKcDFEbFfZt7eV1mTsCSp6SX138owM5+IiCuBo4E+k7CjoyVJrSHaRv7Y2iUiZlRrwEREJ/AS4K7+ylsTliSpdmYC3672C7cBP8zMX/RX2CQsSWoJSV1GR98KPGuw5U3CkqQWEC5bKUlSYUqYhMsXkSRJLcKasCSp+UX9pygNhklYktT00j5hSZIKVMKacPl+FkiS1CKsCUuSWoLN0ZIkFSLqsljHUJmEJUktoYw14fJFJElSi7AmLElqfkHzjI6OiHfXOhBJkkZPkLSN+FFrwz3je2oahSRJLWi4zdHlq9NLktSPpLmWrcyaRiFJ0igr4+jofpNwRKyg72QbwIRRi0iSpFHQUPOEM3NSPQNR39q61zB1xeVMW/FbJq25kc5199PWvYoNY6awqnNflkw+gUemnUp3W2fRoWrQkhlxN7Pjj8xqm8/smM+OcRtjYz1dOY6PrX+y6ABVA21TprPt6R8g2jsAWHnpBay//YaCo1LZDKk5OiK2AU4AXpeZrxidkNTTIXfsxtjuFU853rFxCR0rr2LqyqvYaelXWDDnh6wZt2cBEWqopvAgZ3UcWHQYGmXbvOykzQlYZdCguyhFRAfwcuB1wNHAT4CvjHJcqhrbvYLuGMdj2x7P0snHsqLzIDaMmcK4roXMXPYNdlr6NbZZdzf733c88/f6A91jJhYdsobgyZzFQ91z2SaWMKftuqLDUY107DOX9jlPZ+MTSxgzZXrR4aiqoQZmRcRLgX8AjgJ+B3wXODgzT69TbAIe3u6tPLj9++hq32GL4xvGTuXeWeeyrn1ndn/kHDq7HmTW0nks3N7ZY2W3mu24oOtHPNT9HFayIwAvGvMvJuEmEeM6mXDk/yG7N7L6iouYdOIZRYckqqOjS9gnPFDd/DJgD+DwzDwlMy8BuusTlja5d9bnn5KAe1o44yy6xmwHwLQVl9crLI3AeiZxV/dxmxOwmkvnkSfQts1k1v3pajY++nDR4ajkBmqOPgg4GfhtRNwH/Dcwpi5RafBiLGvG7UH76qV0dC0uOhqppY2dtTvj9n8e3SufZPW1l9I23okkpRHl7BPuN6LMvCkz35eZewAfBZ4FdETEryLC9pUSad/wKAAbxjigXSpMWxsTjjqZiDZWX3ERrF9bdETqJavbGY7kUWuD+lmQmddl5juAWcC5wCE1j0TDMnHNzXSufwCAFRMOLjYYqYWNP/gljJ0+k64H7mb9XX8qOhw1iIEGZj2716EElmTmZVT6i1UCuy/+EFD5hbd4mmPmpCK0TZlO5yFHkRu6WPXbHxYdjvpRxubogfqE/72PY9OqU5ZOzsxbhnKhiNgI3EZlxa2NwDsy8/dDOYe2NPvRLzB15ZUALNruH1nVuV+xAUktapuXvpZo72DN9ZfRvezRosNRP8o4OnqgFbOO7Ot4RMwFzgOeP8RrrcnMA6vnOAr4FPCCIZ5DVVNXXM7uj3wEgJXj9+W+mZ8sOCKpNXU84yDad3sGG59YyprrbSQsqyzpYh1Djigz5wMjXRFiMvA4QES8MCJ+semNiPhyRJwWES+OiIt7HH9pRFw0wus2hYmrb2KfB08l2Mja9tncPucil62UClCZE3wiAKv/58ewoavgiNRohryLUkTswPB2UeqMiJuB8cBM4EVbKX8FcH5EzMjMx4DTgW8O47pNpXPdn9n//hMY272C9WOmc+tuP2ddx+yiw5JaUudhx9A2cTLr/3wrXX+5vehwtBUN1RwdEefx1GQ7DTgUOGsY1+rZHH0I8J2I6LcTMzMzIr4LnBIR36QyIvsNfcR5BnAGwA4zdx5GWI1j3PqHOOC+4+jYuIQNbZO5bbefsmb83kWHJbWstm0rC+V0PO0App193oBlJ778FHj5KQA88ZWP0L182ajHpy011LKVwPxerxNYCrwnM0c08iAzr4+I6cAMYANbNouP7/H8m8AlwFrgR5m5oY9zzQPmAey977Obdp/j9g2PccD9xzG+ayEbo5Pb5/yIlROeVXRYktQwMhsrCR+ZmaeNxkUj4ulUVt9aCjwI7BMR46gk4BcD1wJk5qKIWAR8GHjpaMTSCMZsXM7+95/AhHX30B3t3LHrBTw58fCiw5Ja3uorLmLNdZf2+37bxG2Z9Or/r1L22l/Sde9tAHSvdLtKVQyUhA+o8bU29QlDZZrSGzNzI7AwIn4I3Ar8Gbip1+cuBGZk5h01jqchRPda9nvgNUxaczNJG3ft/DWWTT666LAkAd1PLh3w/Vy75u9llz/uWtKFCnLoY5FH3UBJeEJEPAv67snOzCEtCZOZ/a47nZlnA2f38/bhwFeHcq2mkRvZ569vYMqqawH4y8xPsnTS0bRtXNl3+Wiju821ahvBjLiTcSzf/HpyVP5xDpLZseXG74vzQDYyrq7xSc2mrLsoDZSEZ1FZsKOvqJOtj24esYi4EVgFvHe0r1VG47oeYvryX25+vefi97Pn4vf3W35t+y7c8Iw76xGaRuj4se9it7ZrnnJ8bKznzI4tp89/bt1dPMGcOkUmNa9GS8L3ZuaoJ9qBZOZBRV5fkqTRNOR5wqqfdR27ctUBq4oOQ6Pg613u/dwKupcvY9m/vbPoMFRVxprwQL3Un4qIfXofjIh9I2LGKMYkSVKNjXwbw3pvZXgilXm8vc0GvljzSCRJajEDJeH9M/Oq3gerWxnWevqSJEmjKjNG/Ki1gfqEOwZ4r73WgUiSNFrKOkVpoJrwPRHx8t4HI+IY4L7RC0mSpNorY5/wQDXhfwJ+ERGvBW6sHptLZSOFY2seiSRJLabfmnBm3gPsD1wFzAF2Ba4E3sTwdlGSJKkwjVYTJjPXAd+sLl/5D8BHgPuBn9Q8EkmSRs3oDKwaqYH2E94LOJlK8l0K/ACIzDyyTrFJklQTCXSXcGDWQDXhu4BrgOMy816AiPinukQlSVILGGh09KuAR4DfRcRXI+LF9LOjkiRJZVfGPuGBBmZdnJknAU+nMiDrn4AdIuI/I+JlNY9EkqTRkuVcrGOrOxxn5qrMvDAzj6WyZOXNQP/76UmSpEEZ0i5KmbkM+K/qQ5KkhlHGFbPcylCS1AIabIqSJEnNohHXjpYkSaPImrAkqSXYHC1JUkG6iw6gDyZhSVJLKGNN2D5hSZIKYk1YktT0RmvZyZEyCUuSWoLN0ZIkFaQeGzhExM4R8buIuDMiFkTEWQOVtyYsSVLtbADem5l/iohJwI0RcXlm3tFXYZOwJKn5JXRnHS6TuRhYXH2+IiLuBGYBJmFJUmsqYtnKiJgDPAu4ob8yJmFJkgZvekTM7/F6XmbO610oIiYCPwHenZnL+zuZSViS1BJqNDp6SWbOHahARLRTScAXZuZFA5U1CUuSWkLWoU84IgL4OnBnZn5+a+WdoiRJagFBdw0eg3AYcCrwooi4ufp4eX+FrQlLklQjmXktDH4EmElYktT0knKumGUSliS1hHr0CQ+VSViS1BLKuIGDA7MkSSqINWFJUvOr07KVQ2USliQ1vbIOzLI5WpKkglgTliS1BEdHS5JUkEGueFVXJmFJUksoY03YPmFJkgpiTViS1PSSKOXoaJOwJKn5OU9YkqTi2CcsSZI2syYsSWoJZdzAwSQsSWp6iX3CkiQVpox9wk2VhB9+eBUf/MAfiw5D0kBO/GLREaiG/tbuv7kj0VRJWJKk/lgTliSpAJnQXcLFOpyiJElSQawJS5Jags3RkiQVxCQsSVJByjhP2D5hSZIKYk1YktT0EtzKUJKkQqR9wpIkFcY+YUmStJk1YUlS06v0CRcdxVOZhCVJLaGMSdjmaEmSCmJNWJLUEso4MMskLElqfk5RkiSpGAl0dxcdxVPZJyxJUkGsCUuSWoLN0ZIkFcQkLElSATLLOTraPmFJkgpiTViS1BKyhO3RJmFJUksoYQ42CUuSWoPzhCVJ0mbWhCVJTS9dtlKSpOI4RUmSJG1mTViS1BJsjpYkqSBZwvZok7Akqem5bKUkSdqCNWFJUkuwT1iSpIJ0l7A92iQsSWp6STlrwvYJS5JUEGvCkqTm57KVkiQVJekuYRa2OVqSpIJYE5YktYQs4X7CJmFJUtOrjI62OVqSpPpL6O4e+WNrIuIbEfFoRNw+mLBMwpIk1c63gKMHW9jmaElSS6hHc3RmXh0RcwZb3iTcQI6YdQ0nPO3n7DX1XrZpX8mSNdO5YfHB/Pddr+XhlbOKDk/D4D1tBsmMuJvZ8Udmtc1ndsxnx7iNsbGerhzHx9Y/WXSAotInXMJVK03CjSF5/8Gf5bg9Lt3i6KyJiznxaT/j6Dm/4ZzrPsL1i59XUHwaOu9ps5jCg5zVcWDRYWhrsmb7CU+PiPk9Xs/LzHnDPZlJuAG8YZ8LN/9j/T8PHsm37ziFpWu2Y7/pCzjr2eex08RH+PhhH+NNl81j4YqdC45Wg+E9bU5P5iwe6p7LNrGEOW3XFR2ORseSzJxbq5M5MKvkpo5fxqn7XADAdQ8/j3N+fw5/eWIPnlg3hWsfPox3XfEFVneNZ0L7Gs444GsFR6vB8J42l9VsxwVdP+LT6x7gs+v/wvc3/ID7ul9YdFjqQ+bIH7VmEi65Y+b8hgntawGYd+tbgNji/cWrZnLJX44F4AWzr2Hq+GX1DlFD5D1tLuuZxF3dx7GSHYsORVvR3Z0jfmxNRHwfuB7YOyIeiog3D1TeJFxyh836PQB/XT6be5/Ys88yv1v4AgDGtHVzyMwb6habhsd7KtVfZtbkMYjr/ENmzszM9sycnZlfH6i8Sbjk9pr6ZwDuWPqMfsvctWxvNnRXbuXe0+6pS1waPu+ppE1MwiU2vfMxJrSvAWDRyp36LdfV3cHSNdsBsOvkv9YlNg2P91QqTnaP/FFrJuESmzLu7/MLn1i37YBlH183FYDJHctHNSaNjPdUKk535ogftTZqSTgiVvZ6fVpEfHkrn/loRPxzH8fnDHYdzmYyfuzazc/Xb+wYsOy66vudY9eMakwaGe+ppJ6cJ1xiwd9/dWWvEbRPLfvUZyof76lUnDLuolRIEo6IXYFvADOAx4DTM/OvvcocVC2zGri27kGWwJoNnZufjxuzbsCyHWPWVz8zflRj0sh4T6ViZDKoKUb1Npp9wp0RcfOmB/DxHu99GfhOZh4AXAh8qY/PfxN4V2YeMtBFIuKMiJgfEfO7mmyN1p59hj37EvsyZdwTACxfP3lUY9LIeE+l4rTaYh1rMvPATQ/gnB7vHQJ8r/r8u8DhPT8YEdsCUzLzqh5l+pSZ8zJzbmbObe8YeKBLo1myZgaruyo1p5kTF/dbrr1tPdM7lwDw4PJd6hKbhsd7KqmnsoyO7v37Ivo41pLuefxpAOy73Z39ltl72j2MbauMnb972V51iUvD5z2VipHdOeJHrRWVhH8PnFPcXHQAAAxVSURBVFx9/np69flm5hPAkxFxeI8yLem6hw8FYJfJC9l92/v6LHPkzpUGg43dbVy/+Ll1i03D4z2V6i9rMD2poaYobcW7gNMj4lbgVOCsPsqcDpwfEdcDLTtH41cPvGzzwJy39rGY/47bLOb4PS4B4KqHjuDxtdPqGp+GznsqFaOMNeFRGx2dmRN7vf4W8K3q8weAF/XxmY/2eH4j8Mweb3+0d/lW8PjaaXxnwSm89Zlf4/DZv+djh36Mby84lWVrp7HPdnfw7oPOY0L7WlZ3dVY3A1DZeU+bz4y4k3H8fVGVyfEwUJmSNju2XPt7cR7IRsbVNT6Vl/OEG8B37ng9O01cxHF7XMpLdv0dL9n1d1u8v7qrk3Ou+4j7zjYQ72lzOX7su9it7ZqnHB8b6zmz4wVbHPvcurt4gjl1ikw9jUZNdqRMwg0h+PQfzub3iw7hlXv+nL2n/pkJ7atYsmY6f1j8HL5/10k8vHJW0UFqSLynUl0llDAHm4QbydUPHcHVDx1RdBiqIe9pc/h61+VFh6AGZRKWJDW9xOZoSZIKkq4dLUlSIVpw7WhJkjQAa8KSpJZgc7QkSQVwYJYkSUXJciZh+4QlSSqINWFJUgsYnV2QRsokLElqCWVsjjYJS5KaXlLO0dH2CUuSVBBrwpKk5lfSFbNMwpKkllDGPmGboyVJKog1YUlSC3AXJUmSCpEJ2d1ddBhPYRKWJLWEMg7Msk9YkqSCWBOWJLUE+4QlSSpCZimnKJmEJUlNr6z7CdsnLElSQawJS5JaQnc6RUmSpPpLm6MlSVIP1oQlSU0vcXS0JEmFcZ6wJElFSOgu4drR9glLklQQa8KSpJZgn7AkSQVIknSesCRJBXCesCRJ6smasCSpJZSxJmwSliS1gHTtaEmSipD2CUuSpJ6sCUuSWkKWcMUsk7AkqfnZHC1JknqyJixJagGumCVJUiES6C5hc7RJWJLU/LKcA7PsE5YkqSDWhCVJLSAdHS1JUlEyu0f82JqIODoi7o6IeyPi/Vsrb01YktT86jBPOCLGAOcDLwUeAv4YET/PzDv6+4w1YUmSauNg4N7MvC8z1wP/DbxyoA9YE5YkNb0k6zE6ehawsMfrh4DnDvSByCxfR/VwRcRjwINFx1EH04ElRQehmvKeNp9Wuae7ZuaMooPYmoj4NZV7MlLjgbU9Xs/LzHnVa7wGOCoz31J9fSpwcGa+s7+TNVVNuBH+R6iFiJifmXOLjkO14z1tPt7TcsnMo+twmYeAnXu8ng0sGugD9glLklQbfwSeFhG7RUQHcDLw84E+0FQ1YUmSipKZGyLiHcBlwBjgG5m5YKDPmIQb07yiA1DNeU+bj/e0BWXmpcClgy3fVAOzJElqJPYJS5JUEJOwJEkFMQk3mIiIomOQ1L+ImFJ0DGocJuEGEhHPAd4QEZ1Fx6LaiwgHSja4iJgFXBcRLyo6FjUGk3Bj2QZ4B/CqiBhfdDCqnYjYC/jPiBhXdCwanoiIzHwY+Czw2Yh4XtExqfxMwg0gIvaPiFMz80rgvcBbgNeaiBtfj+6FDqCbytxCNZhqAt401eQBKsvnzouIQ4uLSo3AJNwY9gf+T0S8LjOvBj4KvAkTcTOYXP3v3cAOwEcKjEXDtCkBVxdq+ASV3XOuAL4SEc8vMjaVm0m4xDbVkjLze8CPgFdExCnVGvFHqSTiV9tH3JgiYjbwnYh4c2Z2Uelq2CYidi04NA1SROwdEcf0OPQ04P/PzB8C7wG+AnwhIg4vJECVngNBSqpX8xaZ+f2IWA6cGhFk5gURcQ7wJaAL+EFRsWroImIXKtuefR7454g4gEpNuBN4OvBg7/8HVC4R0Q68CphVvVW/pnL/TgGuyMzuiLgC+Afg0xHx0sxcU2DIKiFXzCq5iPhHYBcqW2edDxxOZVHwSzPzexFxGPBQZrbCFo4NLyLagG2BT1PZd/TfgQCmUOnvfwmwAnh1Zj5SVJwanIjYEXgDMJPKD+E7qSxZeENmviciTgL2A76cmX8rLlKVlUm4ZCJiQmaurj5/F3A88HHgXOAnmfmJ6p6VbwC+W232Usn1rtVWp7C8BrgPuDgz760e3xd4K/D1zLylkGA1oD7u5QwqXUM7U+kL/jPwEypb2O1P5QfVgIv4q3XZJ1wiEfFy4JMRsXNEjKHypT4KmAs8QmXaQ0dm/gj4L+C64qLVUGRmRsQzI+K86usrgO9T6UM8KSJ2rx5fQGUP0pcVFqz61TMBR8RxEXE0sHdmfobKiOiTgF0y83DgNOBwE7AGYhIuiYg4FvgUcGVmLqQyXWU2cCWVJuhXZuZ64E0RcXxm/qI6J1ElFRF7RMSJEXFC9VAXMC0ivlD9x/xq4JfAmcCJETElIrah0jQ96F1YVH8R8TYqLVSHA1+NiA9l5mepTE86MyJenJmrM3NpkXGq/EzCJVDtV3ov8JbM/GlEjK/+2v4Wlb6mCzKzKyJOA84C/GVdctXFN34GHAacHRFvysw7qExf2ZZK9wLALcBNwK8z84nMXAUcY+2pXCJiz4jYttqisT2VroTXZeaHgUOp/Dg+DfgacDtwW3HRqpE4Oroc1lGpJa2tzvt9f0S8gMoAnWVUJv0fAxwIvCoz/1JcqNqaiNgHuBD4QGZeEhGnAJMjYt/MXBAR/wZ8IiKup1LrfXdm3t6jqXN9geGrl4iYCrwdWB8Rn8rMRyNiKdX7lJmPR8Q/AYdl5rci4kuZubHImNU4HJhVAtX5wO+h0g+4L/Bb4FrgDuAE4B7gYqAtMx8rKk4NTnVO6NWZ2VZ9fSvwMLATcFNmnlY9/grg4cy8uahY1b9NP4qq38+jqdR4NwIfAz4JvBR4XmZuiIh3As+jMmCy26llGixrwiVQ/aL/F/B7KoOxfpaZ6wAi4gzgVvuWGkdmXhsRr4iI+6iMfv5xZn48IjqA2yLiw5n5r5n5y4JD1cDGABuoVFZ+FRGTgbOBVZn5gYiYBFxd/ZH1XOD11oA1VNaES6w6Fen9wGttgm48EfFi4DKgIzO7q8feDEzJzH8vNDgNKCKmA/OBg6vNzztRWbXuFmAl8Hhmfioink2lj/+BzLy/uIjVqByYVUIRMTMi3k1laco3moAbU2b+D5V53vdAZXAP8H9x0E7pZeYS4J3AFRGxH/Bd4HuZ+TYqI9e3j4jPAPdm5u9MwBoum6PL6QkqE/5fuWkRBzWmzLw0IrojYjVwP5VBWL8pOi5tXXVQXRdwK/DBzDy/+tY1wDjgiOp/pWGzOVqqg2rT9OTMvLjoWDQ0EfFS4DzguZn5ZI/jm1e3k4bLJCzVkZsyNKbqFMFzgUMyc1nR8ah52Bwt1ZEJuDFVR0d3AL+NiLmVQ95LjZw1YUkapIiYmJkri45DzcMkLElSQZyiJElSQUzCkiQVxCQsSVJBTMKSJBXEKUpSPyJiI5UlJscCd1JZQnR1r+P3A6dm5hMRMada7u4ep/l8Zn4nIh6gsjUlVDYGuAj4l8xcV/3cLzJzv+p1DwY+B+wAJJUdtW4C/rH6+X2q19gI/Bq4C/gslZ2aNnkdsLoaz13A+Or1z8/Mb4/wr0ZSjTg6WupHRKzMzInV5xcCN2bm53sd/zZwT2Z+oncy7XWuB4C5mbkkIiYC84CuzHxjz89FxA7AH4CTM/P66jZ6rwKuycy/9T5X9fVp1dfv6HXNLeKJiN2pJP8vZuY3a/TXJGkEbI6WBucaYM8+jl8PzBrKiarzTM8EToiIab3efjvw7cy8vlo2M/PHmxLwSGTmfVT2rX7XSM8lqTZMwtJWRMRY4Bh67X4UEWOAFwM/73F4j4i4ucfjiL7OmZnLqTRlP63XW/sBNw4jzJN6Xbezn3J/Ap4+jPNLGgX2CUv964yIm6vPrwG+3uv4HCoJ8/Ien/lLZh44yPNHTaKs+EEfzdGjfU1JI2RNWOrfmsw8sPp4Z2au73kc2BXooNKEPCQRMYlKEr+n11sLgINGEPPWPIvKYC1JJWASloapuq3du4B/joj2wX6uOjDrP4CfZubjvd7+MvDGiHhuj/KnRMSOI423OlDrc1S25ZNUAjZHSyOQmTdFxC3AyVSarPfo0YQN8I3M/FL1+e+qo53bgIuBf+njfH+LiJOBz0XE9kA3cDWVUc0DOSkiDu/x+m3Aomo8N/H3KUrnOTJaKg+nKEmSVBCboyVJKohJWJKkgpiEJUkqiElYkqSCmIQlSSqISViSpIKYhCVJKsj/AyDQJtOeA/ldAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(y_test, s_pred, title=\"Confusion Matrix\")\n",
    "np.set_printoptions(precision=1)\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
